<?php

/**
 * ECSHOP 数据采集管理辅助程序
 * ============================================================================
 * * 版权所有 2005-2012 上海商派网络科技有限公司，并保留所有权利。
 * 网站地址: http://www.ecshop.com；
 * ----------------------------------------------------------------------------
 * 这不是一个自由软件！您只能在不用于商业目的的前提下对程序代码进行修改和
 * 使用；不允许对程序代码以任何形式任何目的的再发布。
 * ============================================================================
 * $Author: YunGaZeon $
 * $Blog: http://www.cnblogs.com/GaZeon/ $
 * $Id: lib_collection.php start:2017-07-09 end:2017-8-16 YunGaZeon $
*/

if (!defined('IN_ECS'))
{
    die('Hacking attempt');
}

/*
	运行中立即输出信息
*/
function streamout($str)
{
	echo str_repeat(' ',4096);
	ob_flush();
	flush();
	echo $str;
}

/*
	显示信息后返回指定页面
*/
function collect_goback($url)
{
	streamout('<br/><hr/><h3>----操作完成，3秒后返回上页---</h3><br/><script>setInterval(function(){window.location.href="'.$url.'";},3000);</script>');
}

/*
	通过编号获取字符类型
*/
function getCharset($charcode)
{
	$char='';
	switch ($charcode)
	{
		case 1:
			$char='GB2312';
			break;
		case 2:
			$char='GBK';
			break;
		case 3:
			$char='UTF-8';
			break;
		case 4:
			$char='BIG5';
			break;
		default:
			$char='UTF-8';
	}
	return $char;
}

/*
	字符转义
*/
function filter($data)
{
	if(get_magic_quotes_gpc()===FALSE)
	{
		return is_array($data)?array_map(function($d){
			if(is_array($d))
				return filter($d);
			else
				return addslashes(trim($d));
			
		},$data):addslashes(trim($data));
	}
	else
		return $data;
}

/*
	html字符转义
*/
function htmltrans($data)
{

	return is_array($data)?array_map(function($d){
		if(is_array($d))
			return htmltrans($d);
		else
			return htmlspecialchars(trim($d));
		
	},$data):htmlspecialchars(trim($data));
}

/*
	去除两边空格
*/
function trim_all($data)
{
	return is_array($data)?array_map(function($d){
		if(is_array($d))
			return trim_all($d);
		else
			return trim($d);
		
	},$data):trim($data);
}

/*
	完善补全内容中图片链接
*/
function finish_content_imgurl($cnt,$baseurl='')
{
	if(empty($baseurl))
		$baseurl='http://';
	
	$reg='/(((http:)?\/\/)?([\.\/]?[a-zA-Z0-9\-\_]+)+\.(gif|jpg|jpeg|bmp|png))/';
	preg_match_all($reg,$cnt,$match);
	if(isset($match[1]))
	{
		foreach($match[1] as $m)
		{
			$_url=complet_url($m,$baseurl);
			$cnt=str_replace($m,$_url,$cnt);
		}
	}
	return $cnt;
}

/*
	将保存的规则字符串转化为正则规则
*/
function str2reg($str,$is_limit=TRUE,$extfield=FALSE)
{
	
	$str=str_replace(['[内容]','[除内容]','[字符]','[除字符]','[书写]','[除书写]','[页码]','(*)','[数字]','[标签]','[除标签]'],['<?内容?>','<?除内容?>','<?字符?>','<?除字符?>','<?书写?>','<?除书写?>','<?页码?>','<?页码?>','<?数字?>','<?标签?>','<?除标签?>'],$str);
	if($extfield)
		$str=use_ext_field($str,$extfield);
	//$str=preg_replace_callback('|[\/\\:\.\[\]\'\"]|',function($doc){return "\\".$doc[0];},$str);
	$str = preg_replace_callback('|[\/\\:\.\'\"\(\)\{\}\]\[\-]|',function($doc){return "\\".$doc[0];},$str);
	$str = preg_replace_callback('|(\s+)|',function($doc){return '\s+';},$str);
	$str = str_replace(['<?内容?>','<?除内容?>','<?字符?>','<?除字符?>','<?书写?>','<?除书写?>','<?页码?>','<?数字?>','<?标签?>','<?除标签?>'],['([\S\s]+?)','[\S\s]+?','([^\'\"]+?)','[^\'\"]+?','([^\]\】]+?)','[^\]\】]+?','(\d+?)','(\d+?)','([^\<]+?)','[^\<]+?'],$str);
	
	if($is_limit)
		$str='/'.$str.'/';
	return $str;
}


/*
	获取并格式化用户输入的规则
*/
function getreg($str,$is_limit=TRUE,$extfield=FALSE)
{
	if(strpos($str,'[reg]')===0)
	{
		//$str=trim(substr($str,5));		
		$str=preg_replace('/^\[reg\]\s*[\/\|]?([^\/\|]+)[\/\|]?$/','$1',$str);
		if($extfield)
			$str=use_ext_field($str,$extfield,TRUE);
		return '/'.$str.'/';
	}
	else if(strpos($str,'[key]')===0)
	{
		return explode(',',substr($str,5));
	}
	else
		return str2reg($str,$is_limit,$extfield);
}


/*
	将自定义字段值套入字符串中
*/
function use_ext_field($str,$extfield=FALSE,$is_trans=FALSE)
{
	if(empty($str)||empty($extfield))
		return $str;
	foreach($extfield as $name=>$ext)
	{
		if($is_trans)
			$ext=preg_replace_callback('|[\/\\:\.\'\"\(\)\{\}\]\[\-]|',function($doc){return "\\".$doc[0];},$ext);
		$str=str_replace('['.$name.']',$ext,$str);
	}
	return $str;
}


/*
	移除字符串中的字符
	如果带有[reg]，则移除匹配内容
	否则直接移除。
*/
function data_replace($str,$rule,$replaceto='',$extcfg=FALSE)
{
	if(empty($str))
		return $str;
	
	if(empty($rule)&&empty($extcfg['replaces']))
		return $str;
	
	if(strpos($rule,'[reg]')===0)
	{
		if(!empty($replaceto))
			$replaceto=getreg($replaceto,true,$extcfg['extfield']);
		$str=preg_replace(getreg($rule,true,$extcfg['extfield']),$replaceto,$str);
	}
	else
		$str=str_replace(use_ext_field($rule,$extcfg['extfield']),use_ext_field($replaceto,$extcfg['extfield']),$str);
	
	//替换所有
	if(isset($extcfg['replaces']))
		$str=replace_all($str,$extcfg['replaces']);	

	return $str;
}


/*
	替换所有信息
*/
function replace_all($str,$replaces)
{
	if(empty($replaces))
		return $str;
	foreach($replaces as $r)
	{
		$_froms=explode('<br>',nl2br($r['replace_from'],false));
		
		foreach($_froms as $_f)
		{
			$str=data_replace($str,$_f,$r['replace_to']);
		}
	}	
	return $str;
}


/*
	数据是否包含字符
*/
function data_include($str,$rule,$extfield=FALSE)
{
	if(empty($str))
		return FALSE;
	if(empty($rule))
		return 0;
	if(strpos($str,'[reg]')===0)
	{
		preg_match(getreg($rule),$str,$_m);
		if(!empty($_m))
			return TRUE;
	}
	else
	{
		if(!empty($extfield))
			$rule=use_ext_field($rule,$extfield);
		if(strpos($str,$rule)!==FALSE)
			return TRUE;
	}
	return FALSE;
}

/*
	获取一个指定长度的随机字符串
*/
function getrand($len)
{
	$str='qwertyuiopasdfghjklzxcvbnm1234567890_-@$%&*';
	return substr(str_shuffle($str),$len);
}


/*
	对比父url来补全url
*/
function complet_url($cut_url,$base_url='http://0')
{
	$base=parse_url($base_url);
	$cut=parse_url($cut_url);
	$base['scheme']=empty($base['scheme'])?'http':$base['scheme'];
	if(empty($cut['scheme']))
	{
		$cut_url=trim($cut_url,'\/');
		
		if(empty($cut['host']))
			$cut_url=$base['scheme'].'://'.trim($base['host'],'\/').'/'.$cut_url;
		else
			$cut_url=$base['scheme'].'://'.$cut_url;
	}
	return $cut_url;
}


/*
	curl请求
*/
function http_curl($url,$convert=TRUE,$timeout=0,$charset='UTF-8')
{
	$info=[];
	$info['error']=FALSE;

	$ch=curl_init();
	curl_setopt($ch, CURLOPT_URL, $url);
	curl_setopt($ch, CURLOPT_TIMEOUT, 0);
	//curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
	curl_setopt($ch, CURLOPT_HEADER, 0);
	curl_setopt($ch,CURLOPT_USERAGENT,'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36');
	curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE); // https请求 不验证证书和hosts
	curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, FALSE);
	
	if($timeout)
	{
		curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, 0);
		curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
		ob_start();
		
        if(curl_exec($ch)===FALSE)
		{
			 $info['error']="cUrl Error:".curl_error($ch);
			 curl_close($ch);
			return $info;
		}
		sleep($timeout);
        $info['data'] = ob_get_contents();
        ob_end_clean();
	}
	else
	{
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
		if(($info['data']=curl_exec($ch)) === FALSE)
		{
		   $info['error']="cUrl Error:".curl_error($ch);
		   curl_close($ch);
		   return $info;
		}
	}
	$info['info']=curl_getinfo($ch);
	
	/*
	出现301,302情况，退出
	*/
	if($info['info']['http_code']==301)
	{
		$info['data']=FALSE;
		$info['error']="Content Error:页面永久性移走";
		curl_close($ch);
		return $info;
	}
	
	//if ($convert && $syscharset != $config['sourcecharset'] && $config['sourcetype'] != 4) {
	if ($convert) {
		//检测网页编码
		$encode = mb_detect_encoding($info['data'], ['ASCII','UTF-8','GB2312','GBK','BIG5']); 
		//转换为系统编码
		$info['data'] = iconv($encode, strtoupper($charset).'//TRANSLIT//IGNORE', $info['data']);
	}
	
	curl_close($ch);
	
	return $info;
}

/*
	下载文件
*/
function curl_download($url,$filename,$allow_format=[])
{
	$data=http_curl($url,FALSE);
	if($data['data'])
	{
		//生成图片名
		//$filename=makefilename();
		
		$_dir=dirname($filename);
		if(!is_dir($_dir))
			mkdir($_dir,777,TRUE);
		
		//生成图片
		@file_put_contents($filename,$data['data']);
		
		//获取图片后缀
		$ext=isset(pathinfo($url)['extension'])?pathinfo($url)['extension']:Format::check($filename,$allow_format);
		
		if(empty($ext))
			return FALSE;
			//$ext=Format::check($filename,$allow_format);
		
		//检查图片后缀
		if(!empty($allow_format)&&!in_array($ext,$allow_format))
			return FALSE;
		//完善图片名
		$fullname=$filename . '.' . $ext;

		if(rename($filename,$fullname)===TRUE)
			return $fullname;

	}
	return FALSE;
	
}


/*
	获取页面中的url
*/
function html_get_urls($html,$node)
{
	$url_arr=[];
	if($node['sourcetype']=='4')
	{
		$tnums=preg_match_all(str2reg($node['text_title_sign']),$html,$match_titles);
		$unums=preg_match_all(str2reg($node['text_url_sign']),$html,$match_urls);
		
		if (is_array($match_titles[1]))
		{
			foreach ($match_titles[1] as $k=>$v) {			
				$_url=[];
				$_url['title'] = preg_replace_callback("#\\\u([0-9a-f]+)#i",function($m){return iconv('UCS-2','UTF-8', pack('H4', "\\".$m));},$match_titles[1][$k]);
				
				$_url['url'] = $match_urls[1][$k];
				$url_arr[]=$_url;
			}
		}
		return $url_arr;
	}
	else
	{
		$reg_range='/'.str2reg($node['url_start'],FALSE).'([\s\S]+)'.str2reg($node['url_end'],FALSE).'/U';
		preg_match($reg_range,$html,$cut);
		$reg_url='|<a[\s\S]+href="([\S\s]+)"[\s\S]+>([\s\S]*)</a>|U';
		preg_match_all($reg_url,$cut[0],$urls);
		foreach($urls[1] as $k=>$v)
		{
			if($node['url_contain']!=''&&strpos($v,$node['url_contain'])===FALSE)
				continue;
			if($node['url_except']!=''&&strpos($v,$node['url_except'])!==FALSE)
				continue;
			if($node['url_title_contain']!=''&&strpos($urls[2][$k],$node['url_title_contain'])===FALSE)
				continue;
			if($node['url_title_except']!=''&&strpos($urls[2][$k],$node['url_title_except'])!==FALSE)
				continue;
			if(checkRepeat($url_arr,$v))
				continue;
			
			$_url=[];
			$_url['url']=$v;
			$_url['title']=$urls[2][$k];
			$url_arr[]=$_url;
		}
	}
	return $url_arr;
}


/*
	通过传入URL与配置获取匹配内容
*/
function url_get_content($html,$rule,$attrrule,$extfield,$replaces)
{
	if(empty($html)||empty($rule))
		return FALSE;
	
	//格式化额外字段
	if(!empty($extfield))
	{
		$_f=[];
		foreach($extfield as $e)
		{
			if(($match=getmatch($html,[$e['field_rule_match'],$e['field_rule_include'],$e['field_rule_except'],$e['field_rule_replace'],$e['field_rule_replaceto']]))!==FALSE)
				$_f[$e['field_name']]=$match['match'];
		}
		$extfield=$_f;
	}
	
	$extcfg=[
		'extfield'=>$extfield,
		'replaces'=>$replaces
	];
	

	$data=[];
	
	$data['imgpath']=[];
	//获取商品名
	if(($match=getmatch($html,[$rule['rule_goods_name'],$rule['include_goods_name'],$rule['except_goods_name']
	,$rule['replace_goods_name'],$rule['replaceto_goods_name']],$extcfg))!==FALSE)
	{
		$data['common']['goods_name']=$match['match'];
	}
	
	//获取售价
	if(($match=getmatch($html,[$rule['rule_shop_price'],$rule['include_shop_price'],$rule['except_shop_price']
	,$rule['replace_shop_price'],$rule['replaceto_shop_price']],$extcfg))!==FALSE)
	{
		$data['common']['shop_price']=$match['match'];
	}

	//获取品牌
	if(($match=getmatch($html,[$rule['rule_brand'],$rule['include_brand'],$rule['except_brand']
	,$rule['replace_brand'],$rule['replaceto_brand']],$extcfg))!==FALSE)
	{
		$data['common']['brand']=$match['match'];
	}

	//获取商品重量
	if(($match=getmatch($html,[$rule['rule_goods_weight'],$rule['include_goods_weight']
	,$rule['except_goods_weight'],$rule['replace_goods_weight'],$rule['replaceto_goods_weight']],$extcfg))!==FALSE)
	{
		$data['common']['goods_weight']=$match['match'];
	}
	/*匹配主图图片并下载*/
	if(($match=getmatch($html,[$rule['rule_goods_img'],$rule['include_goods_img']
	,$rule['except_goods_img'],$rule['replace_goods_img'],$rule['replaceto_goods_img']],$extcfg,$rule['url'],FALSE,TRUE))!==FALSE)
	{
		$data['common']['goods_img']=$match['filename'];
		$data['imgpath'][]=$match['filename'];
	}
	
	//获取商品描述
	$_desc=_getmatch_desc($html,$rule,$extcfg);
	$data['common']['goods_desc']=$_desc['data'];
	if(isset($_desc['fullimgs']))
		$data['imgpath']=array_merge($data['imgpath'],$_desc['fullimgs']);

	//获取商品评价
	$_comment=_getmatch_comment($html,$rule,$extcfg);
	if(!empty($_comment['imgs']))
		$data['imgpath']=array_merge($data['imgpath'],$_comment['imgs']);

	$data['comment']=$_comment['data'];

	//获取商品相册
	$_gallery=_getmatch_gallery($html,$rule,$extcfg);
	$_gallery=$rule['is_gallery_download']==1?$_gallery['filename']:$_gallery['match'];
	$data['common']['goods_gallery']=$_gallery;
	$data['imgpath']=array_merge($data['imgpath'],$_gallery);
	
	//匹配产品类型内容
	if(!empty($attrrule))
	{
		foreach($attrrule as $r)
		{
			if(($match=getmatch($html,[$r['rule_content'],$r['rule_include'],$r['rule_except'],$r['rule_replace'],$r['rule_replaceto']],$extcfg))!==FALSE)
			{
				$data['attr'][$r['attrname']]=[$r['attrid'],$match['match']];
				//$data['attr'][$r['attrname']]=[$r['attrid'],htmlspecialchars(addslashes($match['match']))];
			}
		}
	}
	return $data;
}

/*
	测试-通过传入文档与配置获取匹配内容
*/
function TEST_url_get_content($html,$rule,$attrrule,$extfield,$replaces)
{	
	if(empty($html)||empty($rule))
		return FALSE;
	
	//格式化额外字段
	if(!empty($extfield))
	{
		$_f=[];
		foreach($extfield as $e)
		{
			if(($match=getmatch($html,[$e['field_rule_match'],$e['field_rule_include'],$e['field_rule_except'],$e['field_rule_replace'],$e['field_rule_replaceto']]))!==FALSE)
				$_f[$e['field_name']]=$match['match'];
		}
		$extfield=$_f;
	}
	
	$extcfg=[
		'extfield'=>$extfield,
		'replaces'=>$replaces
	];
	//设置所有图片不下载
	$rule['comment_img_download']=0;
	$rule['comment_extimg_download']=0;
	$rule['desc_img_download']=0;
	$rule['is_gallery_download']=0;
	$rule['is_add_comment_user']=0;

	$data=[];
	
	$data['imgpath']=[];
	//获取商品名
	if(($match=getmatch($html,[$rule['rule_goods_name'],$rule['include_goods_name'],$rule['except_goods_name']
	,$rule['replace_goods_name'],$rule['replaceto_goods_name']],$extcfg))!==FALSE)
	{
		$data['common']['goods_name']=$match['match'];
	}
	
	//获取售价
	if(($match=getmatch($html,[$rule['rule_shop_price'],$rule['include_shop_price'],$rule['except_shop_price']
	,$rule['replace_shop_price'],$rule['replaceto_shop_price']],$extcfg))!==FALSE)
	{
		$data['common']['shop_price']=$match['match'];
	}

	//获取品牌
	if(($match=getmatch($html,[$rule['rule_brand'],$rule['include_brand'],$rule['except_brand']
	,$rule['replace_brand'],$rule['replaceto_brand']],$extcfg))!==FALSE)
	{
		$data['common']['brand']=$match['match'];
	}
	
	//获取商品重量
	if(($match=getmatch($html,[$rule['rule_goods_weight'],$rule['include_goods_weight']
	,$rule['except_goods_weight'],$rule['replace_goods_weight'],$rule['replaceto_goods_weight']],$extcfg))!==FALSE)
	{
		$data['common']['goods_weight']=$match['match'];
	}
	/*匹配主图图片并下载*/
	if(($match=getmatch($html,[$rule['rule_goods_img'],$rule['include_goods_img']
	,$rule['except_goods_img'],$rule['replace_goods_img'],$rule['replaceto_goods_img']],$extcfg,$rule['url']))!==FALSE)
	{
		$data['common']['goods_img']=$match['match'];
	}
	
	//获取商品描述
	$_desc=_getmatch_desc($html,$rule,$extcfg);
	$data['common']['goods_desc']=$_desc['data'];


	//获取商品评价
	$_comment=_getmatch_comment($html,$rule,$extcfg);
	$data['comment']=$_comment['data'];

	//获取商品相册
	$data['common']['goods_gallery']=_getmatch_gallery($html,$rule,$extcfg)['match'];

	
	//匹配产品类型内容
	if(!empty($attrrule))
	{
		foreach($attrrule as $r)
		{
			if(($match=getmatch($html,[$r['rule_content'],$r['rule_include'],$r['rule_except'],$r['rule_replace'],$r['rule_replaceto']],$extcfg))!==FALSE)
			{
				//$data['attr'][$r['attrname']]=[$r['attrid'],htmlspecialchars(addslashes($match['match']))];
				$data['attr'][$r['attrname']]=[$r['attrid'],$match['match']];
			}
		}
	}
	return $data;
}


/*
	根据规则及配置获取匹配内容
	返回：array(match匹配内容(字符串或数组),filename下载文件路径(字符串或数组))
*/
function getmatch($html,$rule,$extcfg=FALSE,$baseurl='',$is_multi=FALSE,$is_download=FALSE,$downloadsubdir='')
{
	if(isset($rule)&&!empty($rule))
	{
		$match=$rule[0];
		$include=$rule[1];
		$except=$rule[2];
		$replace=$rule[3];
		$replaceto=$rule[4];
		
		$match_result='';
		//格式化正则规则
		$_reg_match=getreg($match,TRUE,$extcfg['extfield']); 
		//如果是获取单个
		if(empty($is_multi)||$is_multi['mode']===FALSE)
		{
			//如果传入数据是数组，则根据键查找获取数据
			if(is_array($html))
				$match_result=findbykey($_reg_match[0],$html);
			else//如果是字符串，则匹配数据
			{
				preg_match($_reg_match.'U',$html,$match);	
				$match_result=$match[count($match)-1];
			}
			if(empty($match_result))
				return FALSE;
			
			$mts=[];
			//如果该内容不被排除，则往下
			if(!data_include($match_result,$except,$extcfg['extfield'])&&data_include($match_result,$include,$extcfg['extfield'])!==FALSE)
			{
				//根据规则去除数据中指定字符
				$match_result=data_replace($match_result,$replace,$replaceto,$extcfg);
				
				//完善URL
				if(!empty($baseurl))
					$match_result=complet_url($match_result,$baseurl);

				//通过匹配到的链接下载内容
				if($is_download)
				{
					//下载（图片）
					$mts['filename']=curl_download($match_result,makefilename($downloadsubdir),['jpg','bmp','png','gif']);
				}	
				$mts['match']=$match_result;
				return $mts;
			}
		}
		else//匹配获取多个
		{	
			if(is_array($html))//如果传入数据是数组，则根据键查找获取数据
			{
				//如果匹配多层键值，则层层获取
				if(is_array($_reg_match))
					$match_result=findbykeys($html,$_reg_match);
				else
					$match_result=findbykey($_reg_match,$html);
			}
			else//如果是字符串，则匹配数据
			{
				preg_match_all($_reg_match,$html,$match);
				$match_result=$match[count($match)-1];
			}

			if(empty($match_result))
				return FALSE;
			
			/*设置获取数量限制*/
			if(isset($is_multi['max'])&&intval($is_multi['max'])>0)
			{
				if(intval($is_multi['max'])<count($match_result))
					$match_result=array_slice($match_result,0,intval($is_multi['max']));
			}
			
			$mts=[];
			foreach($match_result as $v)
			{
				if(empty($v))
					continue;

				//如果该内容不被排除，则往下
				if(!data_include($v,$except,$extcfg['extfield'])&&data_include($v,$include,$extcfg['extfield'])!==FALSE)
				{
					//根据规则去除数据中指定字符
					$v=data_replace($v,$replace,$replaceto,$extcfg);

					//完善URL
					if(!empty($baseurl))
						$v=complet_url($v,$baseurl);
					
					//通过匹配到的链接下载内容
					if($is_download)
					{
						//下载（图片）
						$mts['filename'][]=curl_download($v,makefilename($downloadsubdir),['jpg','bmp','png','gif']);
					}
					$mts['match'][]=$v;
				}
			}
			if(empty($mts))
				return FALSE;
			
			return $mts;
		}
	}
	return FALSE;
}

/*
	内部，获取匹配的详情(描述)内容
*/
function _getmatch_desc($html,$rule,$extcfg)
{
	$_desc='';
	//商品详情是否延迟加载
	if($rule['is_desc_lazyload'])
	{
		$_url=FALSE;
		//商品详情延迟加载URl获取类型
		if($rule['desc_url_get_type']==='0')//拼接URL
		{
			$_url=use_ext_field($rule['json_desc_lazyload_url'],$extcfg['extfield']);
		}
		else if($rule['desc_url_get_type']==='1')//从页面中正则获取
		{
			if(($async_url=getmatch($html,[$rule['rule_desc_lazyload_url'],$rule['include_desc_lazyload_url']
			,$rule['except_desc_lazyload_url'],$rule['replace_desc_lazyload_url'],$rule['replaceto_desc_lazyload_url']],$extcfg))!==FALSE)
				$_url=$async_url['match'];
		}
		//完善URL
		$_url=complet_url($_url,$rule['url']);
		if($_url)
		{
			//从URL中获取页面数据
			$info=http_curl($_url,getCharset($rule['charset']));
			if($_desc=$info['data'])
			{
				//取得数据为HTML
				if($rule['desc_data_type']==='0')
				{
					if($_desc=getmatch($_desc,[$rule['rule_goods_desc'],$rule['include_goods_desc']
					,$rule['except_desc_url'],$rule['replace_goods_desc'],$rule['replaceto_goods_desc']],$extcfg)!==FALSE)
						$_desc=trim(trim($_desc,'\t'),'\n');
				}
				//取得数据为JSON
				else if($rule['desc_data_type']==='1')
				{
					//去除JSONP
					if($rule['desc_data_jsonp']==='1')
					{
						preg_match('/^[^\(]+\(([\s\S]+)\);$/',$_desc,$_desc_match);
						$_desc=$_desc_match[1];
					}
					//解析JSON为数组
					$_desc=json_decode($_desc,TRUE);
					//如果数据在其中一个键值中，则查找此键取出其数据
					if(!empty($rule['desc_data_key']))
					{
						if(count($_desc)>0)
						{
							$_desc=findbykey($rule['desc_data_key'],$_desc);
						}
					}
					
				}
			}
		}
	}
	else//非延迟加载，在页面内部获取数据
	{
		if(($match=getmatch($html,[$rule['rule_goods_desc'],$rule['include_goods_desc']
		,$rule['except_goods_desc'],$rule['replace_goods_desc']
		,$rule['replaceto_goods_desc']],$extcfg))!==FALSE)
			$_desc=trim(trim($match['match'],'\t'),'\n');
	}
	$_description=[];
	
	//根据规则去除数据中指定字符
	$_desc=data_replace($_desc,$rule['replace_goods_desc'],$rule['replaceto_goods_desc'],$extcfg);

	$_description['data']=$_desc;
	
	if(!empty($_desc))
	{
		/*如果设置下载描述内图片，则查找图片链接并下载*/
		if($rule['desc_img_download']===1)
			$_description=_download_content_img($_desc,$rule);
		else
			//完善补全内容中图片链接
			$_desc=finish_content_imgurl($_desc,$rule['desc_img_url_completion']);
		
		//根据设置的函数处理数据
		if(!empty($_desc))
		{
			if(!empty($rule['desc_data_set']))
			{
				$sets=explode(',',$rule['desc_data_set']);
			}
			$_description['data']=$_desc;
			return $_description;
		}
	}
	return FALSE;
}

/*
	内部：匹配评论内容数组
*/
function _getmatch_comment($html,$rule,$extcfg)
{
	//商品评价是否多页加载
	$comments=[];
	$comments['data']=[];
	$comments['imgs']=[];
	
	//限制最大获取评论数
	$_issuelimit=intval($rule['comment_issuenum']);
	$_current_issue_num=0;
	if($rule['is_comment_multiload'])
	{
		$_url=FALSE;
		//商品评价多页加载URl获取类型
		if($rule['comment_url_get_type']==='0')//拼接URL
		{
			$_url=use_ext_field($rule['json_comment_multiload_url'],$extcfg['extfield']);
		}
		else if($rule['comment_url_get_type']==='1')//页面内正则获取
		{
			if(($async_url=getmatch($html,[$rule['rule_comment_multiload_url'],$rule['include_comment_multiload_url']
			,$rule['except_comment_multiload_url'],$rule['replace_comment_multiload_url'],$rule['replaceto_comment_multiload_url']],$extcfg))!==FALSE)
				$_url=trim($async_url['match']);
		}
		//完善URL
		$_url_raw_page=complet_url($_url,$rule['url']);
	
		//循环页数
		$pn=intval($rule['comment_pagenum'])<=0?1:$rule['comment_pagenum'];
		
		for($_index=1;$_index<=$pn;$_index++)
		{
			//拼接url页数
			$_url=str_replace('(*)',$_index,$_url_raw_page);
			if(empty($_url))
				continue;
			$info=http_curl($_url,getCharset($rule['charset']));
			if($_comment=$info['data'])
			{
				//评论数据类型为HTML处理
				if($rule['comment_data_type']==='0')
				{
					//获取总评论范围
					if($rule['comment_start']&&$rule['comment_end'])
					{
						$reg_range='/'.str2reg($rule['comment_start'],FALSE,$extcfg['extfield']).'([\s\S]+)'.str2reg($rule['comment_end'],FALSE,$extcfg['extfield']).'/U';
						preg_match($reg_range,$html,$_cut);
						if(!empty($_cut[1])) $html=$_cut[1];
					}
					//从单个评价范围配置获取多个评价文档
					$reg_range='/'.str2reg($rule['comment_one_start'],FALSE,$extcfg['extfield']).'([\s\S]+?)'.str2reg($rule['comment_one_end'],FALSE,$extcfg['extfield']).'/U';
					preg_match($reg_range,$html,$_single_cmt);
					if(!empty($_single_cmt[1])) 
						$_comment=$_single_cmt[1];
				}
				//评论数据类型为JSON处理
				else if($rule['comment_data_type']==='1')
				{
					//移除JSONP
					if($rule['comment_data_jsonp']==='1')
					{
						preg_match('/^[^\(]+\(([\s\S]+)\);$/',$_comment,$_comment_match);
						$_comment=$_comment_match[1];
					}
					//解析JSON为数组
					$_comment=json_decode($_comment,TRUE);

					//如果数据在其中一个键值中，则查找此键取出其数据
					if(!empty($rule['comment_data_key']))
					{
						if(count($_comment)>0)
							$_comment=findbykey($rule['comment_data_key'],$_comment);
					}
				}
				if(empty($_comment))
					continue;
				//获取多个评价内容			
				foreach($_comment as $_k=>$cm)
				{
					if($_issuelimit>0&&$_current_issue_num>$_issuelimit)
						break;
					$_cm=_get_one_comment($cm,$rule,$extcfg);
					
					if(!empty($_cm['imgs']))
						$comments['imgs']=array_merge($comments['imgs'],$_cm['imgs']);
					unset($_cm['imgs']);
					$comments['data'][]=$_cm;
					
					++$_current_issue_num;
				}
			}
		}	
	}
	else
	{
		//获取总评论范围
		if($rule['comment_start']&&$rule['comment_end'])
		{
			$reg_range='/'.str2reg($rule['comment_start'],FALSE,$extcfg['extfield']).'([\s\S]+)'.str2reg($rule['comment_end'],FALSE,$extcfg['extfield']).'/U';
			preg_match($reg_range,$html,$_cut);
			if(!empty($_cut[1])) $html=$_cut[1];
		}
		//从单个评价范围配置获取多个评价文档
		$reg_range='/'.str2reg($rule['comment_one_start'],FALSE,$extcfg['extfield']).'([\s\S]+?)'.str2reg($rule['comment_one_end'],FALSE,$extcfg['extfield']).'/U';
		preg_match($reg_range,$html,$_single_cmt);
		if(!empty($_single_cmt[1])) 
			$_single_cmt=$_single_cmt[1];

		//获取多个评价内容
		foreach($_single_cmt as $_k=>$_scmt)
		{
			if($_issuelimit>0&&$_current_issue_num>$_issuelimit)
				break;
			$_cm=_get_one_comment($_scmt,$rule,$extcfg);
			if(!empty($_cm['imgs']))
				$comments['imgs']=array_merge($comments['imgs'],$_cm['imgs']);
			unset($_cm['imgs']);
			$comments['data'][]=$_cm;
			
			++$_current_issue_num;
		}
	}
	return $comments;
}
/*
	内部：获取评论内容数组
*/
function _get_one_comment($html,$rule,$extcfg)
{
	//print_r($html);
	$_cm=[];
	$_cm['imgs']=[];
	//获取评价内容
	if(($match=getmatch($html,[$rule['rule_comment_content']
	,$rule['include_comment_content']
	,$rule['except_comment_content']
	,$rule['replace_comment_content']
	,$rule['replaceto_comment_content']],$extcfg))!==FALSE)
	{
		$_cm['content']=$match['match'];
		/*如果设置下载描述内图片，则查找图片链接并下载*/
		if($rule['comment_img_download']===1)
		{
			$_cmt=_download_content_img($match['match'],$rule,'comment');
		
			if(!empty($_cmt['imgs']))
				$_cm['imgs']=array_merge($_cm['imgs'],$_cmt['fullimgs']);
			
			if(!empty($_cmt['data']))
			{
				$_cm['content']=$_cmt['data'];
				/*通过可选函数处理内容*/
				if(!empty($rule['comment_data_set']))
				{
					$sets=explode(',',$rule['comment_data_set']);
				}
			}
		}
		else
			$_cm['content']=finish_content_imgurl($_cm['content'],$rule['comment_img_url_completion']);
	}
	//获取评价用户名
	if(($match=getmatch($html,[$rule['rule_comment_user']
	,$rule['include_comment_user']
	,$rule['except_comment_user']
	,$rule['replace_comment_user']
	,$rule['replaceto_comment_user']],$extcfg))!==FALSE)
	{
		$_cm['user']=$match['match'];
		//是否生成新用户
		if($rule['is_add_comment_user']===1)
		{
			$sql="SELECT count(id) FROM {$GLOBALS['ecs']->table('users')} WHERE user_name= {$_cm['user']}";
			if($GLOBALS['db']->getOne($sql)<=0)
			{
				$sql="INSERT INTO {$GLOBALS['ecs']->table('users')} (email,user_name,password,sex,reg_time,birthday) VALUES
					'".getrand(8)."@collect.com','{$_cm['user']}','".md5(123)."','".rand(0,1)."','".time()."','".date('Y-M-D',strtotime('-'.rand(12,80)))."'
					)";
				if($GLOBALS['db']->query($sql))
					$_cm['new_user']=$GLOBALS['db']->insert_id();
			}
		}
	
	}
	//获取评价分数
	if(($match=getmatch($html,[$rule['rule_comment_rank']
	,$rule['include_comment_rank']
	,$rule['except_comment_rank']
	,$rule['replace_comment_rank']
	,$rule['replaceto_comment_rank']]))!==FALSE)
		$_cm['rank']=$match['match'];
	
	//获取评价时间
	if(($match=getmatch($html,[$rule['rule_comment_time']
	,$rule['include_comment_time']
	,$rule['except_comment_time']
	,$rule['replace_comment_time']
	,$rule['replaceto_comment_time']]))!==FALSE)
	{
		if($rule['is_timestamp_format'])
			$match['match']=strtotime($match['match']);
		$_cm['time']=$match['match'];
	}
	//获取评价附属图片
	if(($match=getmatch($html,[$rule['rule_comment_extimg']
	,$rule['include_comment_extimg']
	,$rule['except_comment_extimg']
	,$rule['replace_comment_extimg']
	,$rule['replaceto_comment_extimg']],FALSE,empty($rule['comment_img_url_completion'])?'http://':$rule['comment_img_url_completion']
	,TRUE,$rule['comment_extimg_download'],'comment'))!==FALSE)
	{
		$_cm['extimg']=$rule['comment_extimg_download']===1?$match['filename']:$match['match'];
		if(!empty($match['filename']))
			$_cm['imgs']=array_merge($_cm['imgs'],$match['filename']);
	}
	return $_cm;
}

/*
	内部：匹配相册内容数组
*/
function _getmatch_gallery($html,$rule,$extcfg)
{
	//商品图片是否异步加载
	$gallery=[];
	if($rule['is_gallery_async'])
	{
		$_url=FALSE;
		//商品图片多页加载URl获取类型
		if($rule['gallery_url_get_type']==='0')//拼接URL
		{
			$_url=use_ext_field($rule['json_gallery_async_url'],$extcfg['extfield']);
		}
		else if($rule['gallery_url_get_type']==='1')//页面内正则获取
		{
			if(($async_url=getmatch($html,[$rule['rule_gallery_async_url'],$rule['include_gallery_async_url']
			,$rule['except_gallery_async_url'],$rule['replace_gallery_async_url'],$rule['replaceto_gallery_async_url']],$extcfg))!==FALSE)
			{
				$_url=trim($async_url['match']);
			}
		}
		//完善URL
		$_url=complet_url($_url,$rule['url']);

		if(empty($_url))return FALSE;
		
		$info=http_curl($_url,getCharset($rule['charset']));
		if($_gallery=$info['data'])
		{
			//相册数据类型为HTML处理
			if($rule['gallery_data_type']==='0')
			{
				//获取总相册范围
				if($rule['gallery_start']&&$rule['gallery_end'])
				{
					$reg_range='/'.str2reg($rule['gallery_start'],FALSE,$extcfg['extfield']).'([\s\S]+)'.str2reg($rule['gallery_end'],FALSE,$extcfg['extfield']).'/U';
					preg_match($reg_range,$html,$_cut);
					if(!empty($_cut[1])) $html=$_cut[1];
				}
				
				//获取多个图片并根据设置下载图片
				if(($match=getmatch($html,[$rule['rule_goods_gallery']
				,$rule['include_goods_gallery'],$rule['except_goods_gallery']
				,$rule['replace_goods_gallery'],$rule['replaceto_goods_gallery']],$extcfg,$rule['url'],['max'=>$rule['gallery_num_limit']],$rule['is_gallery_download']))!==FALSE)
				{
					$gallery=$match;
				}
				
			}
			//相册数据类型为JSON处理
			else if($rule['gallery_data_type']==='1')
			{
				//移除JSONP
				if($rule['gallery_data_jsonp']==='1')
				{
					preg_match('/^[^\(]+\(([\s\S]+)\);$/',$_gallery_arr,$_gallery_match);
					$_gallery_arr=$_gallery_match[1];
				}
				//解析JSON为数组
				$_gallery_arr=json_decode($_gallery_arr,TRUE);
				
				if(empty($_gallery_arr))return FALSE;
				
				$_gallery=[];
				
				//如果数据在其中一个键值中，则查找此键取出其数据
				if(!empty($rule['gallery_data_key']))
					$_gallery_arr=findbykey($rule['gallery_data_key'],$_gallery_arr);
				
				//根据相册存储类型配置获取相册数据
				switch($rule['gallery_src_type'])
				{
					//单层多条
					case '0':
						$_gallery=$_gallery_arr;
						break;
					//单层单条
					case '1':
						empty($rule['gallery_src_type_separator'])?$rule['gallery_src_type_separator']=',':'';
						$_gallery=explode($rule['gallery_src_type_separator'],$_gallery_arr);
						break;
					//多层多条
					case '2':
						foreach($_gallery_arr as $g)
							$_gallery[]=findbykey($rule['gallery_src_type_key'],$g);
						break;
					default:
						return FALSE;
				}		
				
				//检查图片数量限制
				if(intval($rule['gallery_num_limit'])>0&&intval($rule['gallery_num_limit'])>$_gallery)
					$_gallery=array_slice($_gallery,0,intval($rule['gallery_num_limit']));
						
				
				foreach($_gallery as $k=>$_g)
				{
					//如果不包含字符，则跳过
					if(data_include($_g,$rule['include_goods_gallery'],$extcfg['extfield'])===FALSE)	
						continue;
					
					//排除存在不需要的数据
					if(data_include($_g,$rule['except_goods_gallery'],$extcfg['extfield'])===TRUE)	
						continue;
					
					//替换数据
					$_src=data_replace($_g,$rule['replace_goods_gallery'],$rule['replaceto_goods_gallery'],$extcfg);
					if(empty($_src))continue;
					
					//完善URL
					$_src=complet_url($_src,empty($rule['gallery_img_url_completion'])?'http://':$rule['gallery_img_url_completion']);		
					
					$gallery['match'][]=$_src;
					
					//根据配置下载图片
					if($rule['is_gallery_download'])
					{
						if(($_dsrc=curl_download($_src,makefilename('gallery'),['jpg','png','bmp','gif'])!==FALSE))
						{
							$gallery['filename'][]=$_dsrc;
						}							
					}
				}	
			}
			if(empty($gallery)) return FALSE;
		}
	}
	else
	{
		//获取总相册范围
		if($rule['gallery_start']&&$rule['gallery_end'])
		{
			$reg_range='/'.str2reg($rule['gallery_start'],FALSE,$extcfg['extfield']).'([\s\S]+)'.str2reg($rule['gallery_end'],FALSE,$extcfg['extfield']).'/U';
			preg_match($reg_range,$html,$_cut);
			//($_cut);
			if(!empty($_cut[1])) $html=$_cut[1];
			//print_r($html);
		}
		
		//获取多个图片并根据设置下载图片
		if(($match=getmatch($html,[$rule['rule_goods_gallery']
		,$rule['include_goods_gallery'],$rule['except_goods_gallery']
		,$rule['replace_goods_gallery'],$rule['replaceto_goods_gallery']],$extcfg,$rule['url'],['max'=>$rule['gallery_num_limit']],$rule['is_gallery_download'],'gallery'))!==FALSE)
		{
			$gallery=$match;
		}
	}
	return $gallery;
}

/*
*	内部:下载内容中的图片并将链接替换为本地路径
*/
function _download_content_img($html,$noderule,$type='desc')
{
	$info=[];
	$src_name=explode(',',$noderule['img_src_names']);
	$srcnames='?<=\b'.implode('\b|\b',$src_name).'\b';
	
	if(empty($imgrule))
		$imgrule='/<img[\s\S]+('.$srcnames.')=[\'\"]([^\'\"]+?)[\'\"][\s\S]*[\/]?>/U';
	
	preg_match_all($imgrule,$html,$imgs);
	$info['imgs']=[];
	$info['fullimgs']=[];

	foreach($imgs[1] as $k=>$img)
	{
		if(in_array($img,$info['imgs']))
			continue;
		$dir= ROOT_PATH . IMAGE_DIR . '/' . date('Ymd') . '/'.$type.'/';

		$_url=complet_url($img,$noderule[$type.'_img_url_completion']);
	
		$urlinfo=pathinfo($_url);
		$filename=$dir.$urlinfo['filename'].time();//.'.'.$urlinfo['extension'];
		$filename=curl_download($_url,$filename,['jpg','bmp','gif','png']);
		if($filename)
		{
			$info['fullimgs'][]=$filename;
			$filename=str_replace(ROOT_PATH,'',$filename);
			$info['imgs'][]=$filename;
			//$html=str_replace($img,$filename,$html);
			
			$new_img=str_replace($img,$filename,$imgs[0][$k]);//先替换整条img
			$new_img=str_replace($src_name,'src',$new_img);//再替换src
			
			$html=str_replace($imgs[0][$k],$new_img,$html);//再替换到html
		}
	}
	$info['data']=$html;
	return $info;
}

/*
	在一个不确定维度的数组中根据匹配的规则获取值
*/
function findbyreg($reg,$arr)
{
	static $_reg_str=FALSE;
	foreach($arr as $v)
	{
		preg_match($reg,$v,$_m);
		if(!empty($_m))
			return $_m[0];
		else
		{
			if(is_array($v))
				$_reg_str=findbyreg($reg,$v);
		}	
	}
	return $_reg_str;
}

/*
	在一个不确定维度的数组中根据键获取值
*/
function findbykey($key,$arr)
{
	$_key_str=FALSE;
	foreach($arr as $k=>$v)
	{
		if($k===$key)
			return $v;
		else
		{
			if(is_array($v))
				$_key_str=findbykey($key,$v);
		}	
	}
	return $_key_str;
}

/*
	检查数组中是否有该内容
*/
function checkRepeat($arr,$find)
{
	foreach($arr as $v)
	{
		if($v['url']==$find)
			return TRUE;
	}
	return FALSE;
}
/*
	在一个不确定维度的数组通过通过层级键获取多个目标值数组
	data：数组
	keys:层级键数组
	return 目标值数组
*/
function findbykeys($data,$keys,$_lv=0,&$_findbykeys_arr=[])
{
	if($is_first)
		$_findbykeys_arr=[];
	
	if($_lv>(count($keys)-1))
		return $arr;

	foreach($data as $k=>$v)
	{
		if($k===$keys[$_lv])
		{
			if($_lv==(count($keys)-1)&&!in_array($v,$_findbykeys_arr))
				$_findbykeys_arr[]=$v;
			else
			{
				if(is_array($v))
					findbykeys($v,$keys,++$_lv,$_findbykeys_arr);
			}
		}
		else
		{
			if(is_array($v))
				findbykeys($v,$keys,$_lv,$_findbykeys_arr);
		}
	}
	return $_findbykeys_arr;
}

/*
	生成文件名
*/
function makefilename($subdir='collection_temp',$img_name='',$ext='')
{
	if(empty($subdir))
		$subdir='collection_temp';
	
	$subdir=trim($subdir,'/');
	/* 创建当月目录 */
	$dir = ROOT_PATH . IMAGE_DIR . '/' . date('Ymd') . '/';

	/* 如果目标目录不存在，则创建它 */
	if (!file_exists($dir))
	{
		if (!make_dir($dir))
		{
			/* 创建目录失败 */
			return FALSE;
		}
		make_dir($dir.'goods_img/');
		make_dir($dir.'source_img/');
		make_dir($dir.'thumb_img/');
		make_dir($dir.'collection_temp/');
	}
	if (empty($img_name))
	{
		if(!empty($ext))
			$ext ='.' . $ext;
		
		$image = new cls_image();
		$img_name = $dir .$subdir.'/'. $image->unique_name($dir) . $ext;
	}
	return $img_name;
}

/*
	生成缩略图
*/
function makeImage($imgpath,$takegallery=FALSE,$is_gallery=FALSE)
{
	$image = new cls_image();
	// 如果系统支持GD，缩放商品图片，且给商品图片和相册图片加水印
	if (!empty($imgpath) && $image->gd_version() > 0 )
	{
		global $_CFG;
		if(is_string($imgpath))
			$imgpath=[$imgpath];
		$imgs=[];
		foreach($imgpath as $k=>$img)
		{
			if(empty($img))
				continue;
	
			$dir=trim(str_replace(['collection_temp','gallery'],'',dirname($img)),'\/').'/';
			$org=str_replace('collection_temp','source_img',$img);
			$imgs[$k]['original']=str_replace(ROOT_PATH,'',$org);
			
			if(!$is_gallery)
			{
				if(!is_dir($dir.'source_img/'))
					mkdir($dir.'source_img/');
				touch($org);
				copy($img,$org);
				@unlink($img);
			}
			
			$subdir=$is_gallery?'gallery/':'goods_img/';
			//商品标准图
			if ($_CFG['image_width'] != 0 || $_CFG['image_height'] != 0)
			{
				$imgs[$k]['img'] = $image->make_thumb($org, $GLOBALS['_CFG']['image_width'],  $GLOBALS['_CFG']['image_height'],$dir.$subdir);
			}

			// 加水印img
			if (intval($_CFG['watermark_place']) > 0 && !empty($GLOBALS['_CFG']['watermark']))
			{
				$image->add_watermark('../'.$imgs[$k]['img'],'',$GLOBALS['_CFG']['watermark'], $GLOBALS['_CFG']['watermark_place'], $GLOBALS['_CFG']['watermark_alpha'],$dir.$subdir);
			
				/* 添加判断是否自动生成相册图片 */
				if ($_CFG['auto_generate_gallery']&&$takegallery)
				{
					$imgs[$k]['gallery']=$image->add_watermark('../'.$imgs[$k]['img'],'',$GLOBALS['_CFG']['watermark'], $GLOBALS['_CFG']['watermark_place'], $GLOBALS['_CFG']['watermark_alpha'],$dir.'gallery/');
				}
			}
			
			// 相册缩略图thumb
			/* 添加判断是否自动生成相册图片 */
			if ($_CFG['thumb_width'] != 0 || $_CFG['thumb_height'] != 0)
			{
				$subdir=$is_gallery?'gallery/':'thumb_img/';
				$imgs[$k]['thumb'] = $image->make_thumb('../'.$imgs[$k]['img'], $GLOBALS['_CFG']['thumb_width'],  $GLOBALS['_CFG']['thumb_height'],$dir.$subdir);
			}
		}
		return $imgs;
	}
	return FALSE;
}

/* 
	获得采集内容列表 
*/
function get_collection_content_list($is_first=false)
{
	//取得上次的过滤条件
    $result = $is_first ? FALSE : get_filter();

    if ($result === FALSE)
    {
        $filter = array();
		$filter['act']='content_query';
        $filter['keyword']    = empty($_REQUEST['keyword']) ? '' : trim($_REQUEST['keyword']);

        $filter['is_import'] = $_REQUEST['is_import']<0 ? '' : $_REQUEST['is_import'];
        $filter['sort_by']    = empty($_REQUEST['sort_by']) ? 'a.id' : trim($_REQUEST['sort_by']);
        $filter['sort_order'] = empty($_REQUEST['sort_order']) ? 'DESC' : trim($_REQUEST['sort_order']);
		
		$filter['nodeid'] = empty($_REQUEST['cid']) ? 0 : intval($_REQUEST['cid']);
		$filter['cat_id'] = empty($_REQUEST['cat_id']) ? 0 : intval($_REQUEST['cat_id']);

        $where = '';
		
		if($filter['nodeid']!==0)
			$where .= " AND a.nodeid = " . $filter['nodeid'];
        
        if (!empty($filter['keyword']))
        {
            $where .= " AND a.title LIKE '%" . mysql_like_quote($filter['keyword']) . "%'";
        }
        if ($filter['is_import']!='')
        {
            $where .= " AND a.is_import = '" . $filter['is_import']."'";
        }
		 if ($filter['cat_id']!==0)
        {
            $where .= " AND b.catid = '" . $filter['cat_id']."'";
        }

        /* 采集总数 */
        $sql = 'SELECT COUNT(*) FROM ' .$GLOBALS['ecs']->table('collection_content') .' a 
				LEFT JOIN '.$GLOBALS['ecs']->table('collection_node').' b on a.nodeid=b.id
				WHERE 1 ' .$where;
        $filter['record_count'] = $GLOBALS['db']->getOne($sql);
        $filter = page_and_size($filter);

		//获取采集数据
		$sql = "select a.*,b.name as nodename,c.url,c.sign FROM {$GLOBALS['ecs']->table('collection_content')} a 
			LEFT JOIN {$GLOBALS['ecs']->table('collection_node')} b on a.nodeid=b.id
			LEFT JOIN {$GLOBALS['ecs']->table('collection_site')} c on c.id=a.siteid 
			WHERE 1 ".
			$where. ' ORDER by '.$filter['sort_by'].' '.$filter['sort_order'];

        $filter['keyword'] = stripslashes($filter['keyword']);
        set_filter($filter, $sql);
    }
    else
    {
        $sql    = $result['sql'];
        $filter = $result['filter'];
    }
    $arr = array();
    $res = $GLOBALS['db']->selectLimit($sql, $filter['page_size'], $filter['start']);

    while ($rows = $GLOBALS['db']->fetchRow($res))
    {
        ///$rows['date'] = local_date($GLOBALS['_CFG']['time_format'], $rows['add_time']);
		$rows['common_content']=htmltrans(json_decode($rows['common_content'],TRUE));
		if(!empty($rows['attr_content']))
			$rows['attr_content']=htmltrans(json_decode($rows['attr_content'],TRUE));	
		if(!empty($rows['common_content']['goods_gallery']))
			$rows['common_content']['goods_gallery']=htmltrans(implode('<br>',$rows['common_content']['goods_gallery']));
		if(!empty($rows['comment_info']))
			$rows['comment_info']=htmltrans(json_decode($rows['comment_info'],TRUE));
		$arr[] = $rows;
    }
    return array('arr' => $arr, 'filter' => $filter, 'page_count' => $filter['page_count'], 'record_count' => $filter['record_count']);
}

/*
	获得采集内容列表 
*/
function get_node_list($is_first=false)
{
	//取得上次的过滤条件
    $result = $is_first?FALSE:get_filter();
    if ($result === FALSE)
    {
        $filter = array();
		$filter['act']='node_query';
        $filter['keyword']    = empty($_REQUEST['keyword']) ? '' : trim($_REQUEST['keyword']);
		$filter['cat_id']    = empty($_REQUEST['cat_id']) ? '' : trim($_REQUEST['cat_id']);
        $filter['sort_by']    = empty($_REQUEST['sort_by']) ? 'a.id' : trim($_REQUEST['sort_by']);
        $filter['sort_order'] = empty($_REQUEST['sort_order']) ? 'DESC' : trim($_REQUEST['sort_order']);
		
        $where = '';
        if (!empty($filter['keyword']))
        {
            $where .= " AND a.name LIKE '%" . mysql_like_quote($filter['keyword']) . "%'";
        }
        if ($filter['cat_id']!='')
        {
            $where .= " AND b.cat_id = '" . $filter['cat_id']."'";
        }

        /* 采集总数 */
        $sql = 'SELECT COUNT(*) FROM ' .$GLOBALS['ecs']->table('collection_node') ." a "
				."LEFT JOIN ".$GLOBALS['ecs']->table('goods_type')." b on a.catid=b.cat_id"	
				." WHERE 1".$where;
        $filter['record_count'] = $GLOBALS['db']->getOne($sql);

        $filter = page_and_size($filter);

		//获取采集数据
		$sql = "select a.id,a.name,a.lastdate FROM " . $GLOBALS['ecs']->table('collection_node') ." a "
			." LEFT JOIN ".$GLOBALS['ecs']->table('goods_type')." b on a.catid=b.cat_id "
			." WHERE 1 ".$where
			.' ORDER by '.$filter['sort_by'].' '.$filter['sort_order'];

        $filter['keyword'] = stripslashes($filter['keyword']);
        set_filter($filter, $sql);
    }
    else
    {
        $sql    = $result['sql'];
        $filter = $result['filter'];
    }
    $arr = array();
    $res = $GLOBALS['db']->selectLimit($sql, $filter['page_size'], $filter['start']);

    while ($rows = $GLOBALS['db']->fetchRow($res))
	{
		$rows['lastdate']=date('Y-m-d H:i:s',$rows['lastdate']);
		$sql='select count(*) from '.$GLOBALS['ecs']->table('collection_site').' where nodeid='.$rows['id'];
		$rows['site_num']=$GLOBALS['db']->getOne($sql);
		
		$sql='select count(*) from '.$GLOBALS['ecs']->table('collection_site').' where nodeid='.$rows['id'].' and is_read = \'0\'';
		$rows['site_no_read']=$GLOBALS['db']->getOne($sql);
		
		$sql='select count(*) from '.$GLOBALS['ecs']->table('collection_content').' where nodeid='.$rows['id'];
		$rows['cnt_num']=$GLOBALS['db']->getOne($sql);
		
		$sql='select count(*) from '.$GLOBALS['ecs']->table('collection_content').' where nodeid='.$rows['id'].' and is_import = \'0\'';
		$rows['cnt_no_implode']=$GLOBALS['db']->getOne($sql);
		
        $arr[] = $rows;
    }
    return array('arr' => $arr, 'filter' => $filter, 'page_count' => $filter['page_count'], 'record_count' => $filter['record_count']);
}

